import os.path
import time
import requests
from lxml import etree

def get_html(url):
    '''
    通过传递url 返回对应的tree对象
    :param url: 要请求的url地址
    :return: xpth -- obj对象
    '''
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
    }
    res = requests.get(url, headers=headers)
    tree = etree.HTML(res.content.decode())
    return tree


def get_book_con(tree):
    '''
    获取书的名称和url
    :return: {'三国演义': 'https://www.shicimingju.com/book/sanguoyanyi.html'...}
    '''
    # 返回每本书的超链接
    a_list = tree.xpath('//div[@class="book-item"]/h3/a')
    # print(a_list)
    book_item = {}
    for a in a_list:
        # 获取书的超链接 拼接完整的超链接
        book_url = 'https://www.shicimingju.com' + a.xpath('./@href')[0]
        # 获取书名
        book_name = a.xpath('./text()')[0]
        book_item[book_name] = book_url
    return book_item


def get_book_mulu(tree):
    """
    获取书的章节的名称和url
    :param tree: tree对象
    :return: {'第一回...': https://www.shicimingju.com/book/sanguoyanyi/1.html}
    """
    li_list = tree.xpath('//div[@class="book-mulu"]/ul/li')
    # print(li_list)
    book_mulu = {}
    for li in li_list:
        # 章节的url
        mulu_url = 'https://www.shicimingju.com' + li.xpath('./a/@href')[0]
        # 章节的名称
        mulu_name = li.xpath('./a/text()')[0]
        book_mulu[mulu_name] = mulu_url
    return book_mulu


def get_con(book_name, mulu_name, tree):
    '''
        获取章节内容 并下载
    :param book_name: 传入的书名
    :param mulu_name: 传入的章节名称
    :param tree: 传入的当前内容的tree对象
    :return: None
    '''
    if not os.path.exists(book_name):
        os.makedirs(book_name)  # 递归创建
    # 拼接新路径 用于存储章节内容
    path = os.path.join(book_name, mulu_name + '.txt')
    text = ''.join(tree.xpath('//div[@class="card bookmark-list"]//text()'))
    # print(text)
    with open(path, 'w', encoding='utf-8') as f:
        f.write(text.replace(' ', ''))
    print(book_name, mulu_name, '下载完成...')

# 循环获取四大名著的书名和url
for book_name, book_url in get_book_con(get_html('https://www.shicimingju.com/bookmark/sidamingzhu.html')).items():
    # 循环获取四大名著的书的章节名称和url
    for mulu_name, mulu_url in get_book_mulu(get_html(book_url)).items():
        # 执行下载
        get_con(book_name, mulu_name, get_html(mulu_url))
        # 给一个自省时间 防止干崩网址
        time.sleep(1)


'''
三国演义/   
    第一回 张飞大战李逵.txt
    第一回 张飞大战李逵.txt
    第一回 张飞大战李逵.txt
    ...
水浒传/   
    第一回 宋江智擒唐僧.txt
    第一回 宋江智擒唐僧.txt
    第一回 宋江智擒唐僧.txt
    ...
'''